#爬取豆瓣图书top250
import os,time,requests,json
from bs4 import BeautifulSoup
from urllib.request import urlretrieve

def getPage(offset):
    '''获取一页的网页信息'''
    headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.26 Safari/537.36 Core/1.63.6788.400 QQBrowser/10.3.2767.400',}
    url = 'https://book.douban.com/top250?start='+str(offset)
    try:
        res = requests.get(url,headers=headers)
        if res.status_code == 200:
            return res.text
        else:
            print('没有爬取到数据')
            return None
    except Exception as e:
        print(e)
        return None

def parsePage(html):
    '''解析图书信息'''
    #解析html文档，获取BeautifulSoup对象
    soup = BeautifulSoup(html,'lxml')
    #获取网页中的所有标签，并遍历输出标签名
    items = soup.find_all(name='table',attrs={'width':'100%'})
    #遍历封装数据并返回
    try:
        for item in items:
            yield{
                'image':item.find(name='img',attrs={"width":"90"}).attrs['src'],
                'title':item.select('div.pl2 a')[0].attrs['title'],
                'author':item.find(name='p',attrs={'class':'pl'}).get_text(),
                'socre':item.find(name='span',attrs={"class":"rating_nums"}).get_text(),
                'words':item.find(name='span',attrs={"class":"inq"}).get_text(),
            }
    except Exception as e:
        # print(e)
        return None

def writeFile(item):
    '''储存图书信息'''
    #获取图片，传入的item是一个字典，值是字符串
    imgurl = item.get('image')
    title = item.get('title')
    urlretrieve(imgurl,'./images/douban/bs4/'+title+'.jpg')
    #将图书详细信息写入文件
    with open('./douban_bs4.txt','a',encoding='utf-8') as f:
        f.write(json.dumps(item,ensure_ascii=False)+'\n')

def main(offset):
    '''主函数'''
    #判断当前路径是否存在images文件夹
    if not os.path.exists('./images/douban/bs4'):
        os.makedirs('./images/douban/bs4')
    html = getPage(offset)
    items = parsePage(html)
    for item in items:
        # print(item)
        writeFile(item)

if __name__ == '__main__':
    # main(0)
    for i in range(10):
        offset = i*25
        main(offset)
        print('第%s条数据完成。'%((i+1)*25))
        time.sleep(1)